'''
断点续爬 从要爬取的列表中爬取 成功删除 失败不删除
'''


import random
import redis
import requests


def get_url():
    client = redis.Redis(host='localhost', port=6379, db=0)
    url = client.lpop('requested_urls')
    client.close()
    return url.decode()

def rollback_url(url):
    client=redis.Redis(host='localhost', port=6379, db=0)
    client.lpush('requested_urls', url)
    client.close()

def request_url(url):
    try:
        res = requests.get(url,timeout=1)
        print(res.json())
    except Exception as e:
        print(f'爬取{url}失败 原因{e}')
        rollback_url(url)

def main():
    for i in range(10):
        url = get_url()
        request_url(url)
        break_num = random.randint(1,5)
        if break_num == 3:
            break

if __name__ == '__main__':
    main()




